<html><!-- Created using the cpp_pretty_printer from the dlib C++ library.  See http://dlib.net for updates. --><head><title>dlib C++ Library - optimization.cpp</title></head><body bgcolor='white'><pre>
<font color='#009900'>// Copyright (C) 2008  Davis E. King (davis@dlib.net)
</font><font color='#009900'>// License: Boost Software License   See LICENSE.txt for the full license.
</font>

<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='optimization_test_functions.h.html'>optimization_test_functions.h</a>"
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>dlib<font color='#5555FF'>/</font>optimization.h<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>dlib<font color='#5555FF'>/</font>statistics.h<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>sstream<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>string<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>cstdlib<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>ctime<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>vector<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../stl_checked.h.html'>../stl_checked.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../array.h.html'>../array.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../rand.h.html'>../rand.h</a>"

<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='tester.h.html'>tester.h</a>"


<font color='#0000FF'>namespace</font>  
<b>{</b>

    <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> test;
    <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib;
    <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> std;

    logger <b><a name='dlog'></a>dlog</b><font face='Lucida Console'>(</font>"<font color='#CC0000'>test.optimization</font>"<font face='Lucida Console'>)</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>bool</u></font> <b><a name='approx_equal'></a>approx_equal</b> <font face='Lucida Console'>(</font>
        <font color='#0000FF'><u>double</u></font> a,
        <font color='#0000FF'><u>double</u></font> b
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>return</font> std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>a <font color='#5555FF'>-</font> b<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>100</font><font color='#5555FF'>*</font>std::numeric_limits<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>epsilon</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>long</u></font> total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;


    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'><u>double</u></font> <b><a name='apq'></a>apq</b> <font face='Lucida Console'>(</font> <font color='#0000FF'>const</font> T<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>DLIB_ASSERT</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> x.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font>,"<font color='#CC0000'></font>"<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>COMPILE_TIME_ASSERT</font><font face='Lucida Console'>(</font>is_matrix<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font>::value<font face='Lucida Console'>)</font>;
        <font color='#0000FF'><u>double</u></font> temp <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> x.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
        <b>{</b>
            temp <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>r<font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>r<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>r<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>total_count;

        <font color='#0000FF'>return</font> temp <font color='#5555FF'>+</font> <font color='#979000'>1</font><font color='#5555FF'>/</font><font color='#979000'>100.0</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
    T <b><a name='der_apq'></a>der_apq</b> <font face='Lucida Console'>(</font> <font color='#0000FF'>const</font> T<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>DLIB_ASSERT</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> x.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font>,"<font color='#CC0000'></font>"<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>COMPILE_TIME_ASSERT</font><font face='Lucida Console'>(</font>is_matrix<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font>::value<font face='Lucida Console'>)</font>;
        T <font color='#BB00BB'>temp</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> x.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>temp</font><font face='Lucida Console'>(</font>r<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>2</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font>r<font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>r<font face='Lucida Console'>)</font> ;
        <b>}</b>

        <font color='#BB00BB'>temp</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>/</font><font color='#979000'>50.0</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>temp</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>/</font><font color='#979000'>50.0</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>total_count;

        <font color='#0000FF'>return</font> temp;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#009900'>// Rosenbrock's function.  minimum at (1,1)
</font>    <font color='#0000FF'><u>double</u></font> <b><a name='rosen'></a>rosen</b> <font face='Lucida Console'>(</font> <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>total_count;
        <font color='#0000FF'>return</font> <font color='#979000'>100</font><font color='#5555FF'>*</font><font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font><font color='#979000'>1</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font>;
    <b>}</b>

    matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> <b><a name='der_rosen'></a>der_rosen</b> <font face='Lucida Console'>(</font> <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>total_count;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> res;
        <font color='#BB00BB'>res</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>400</font><font color='#5555FF'>*</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#979000'>2</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>res</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>200</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>return</font> res;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#009900'>// negative of Rosenbrock's function.  minimum at (1,1)
</font>    <font color='#0000FF'><u>double</u></font> <b><a name='neg_rosen'></a>neg_rosen</b> <font face='Lucida Console'>(</font> <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>total_count;
        <font color='#0000FF'>return</font> <font color='#5555FF'>-</font><font face='Lucida Console'>(</font><font color='#979000'>100</font><font color='#5555FF'>*</font><font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font><font color='#979000'>1</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
    <b>}</b>

    matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> <b><a name='der_neg_rosen'></a>der_neg_rosen</b> <font face='Lucida Console'>(</font> <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>total_count;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> res;
        <font color='#BB00BB'>res</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>400</font><font color='#5555FF'>*</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#979000'>2</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>res</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>200</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>return</font> <font color='#5555FF'>-</font>res;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>double</u></font> <b><a name='simple'></a>simple</b> <font face='Lucida Console'>(</font> <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>total_count;
        <font color='#0000FF'>return</font> <font color='#979000'>10</font><font color='#5555FF'>*</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
    <b>}</b>

    matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> <b><a name='der_simple'></a>der_simple</b> <font face='Lucida Console'>(</font> <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>total_count;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> res;
        <font color='#BB00BB'>res</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>20</font><font color='#5555FF'>*</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>res</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>2</font><font color='#5555FF'>*</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>return</font> res;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>double</u></font> <b><a name='powell'></a>powell</b> <font face='Lucida Console'>(</font> <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>4</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>total_count;
        <font color='#0000FF'>return</font> <font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>10</font><font color='#5555FF'>*</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font>
            <font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font><font color='#979000'>5.0</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>3</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> 
            <font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#979000'>2</font><font color='#5555FF'>*</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#979000'>2</font><font color='#5555FF'>*</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font>
            <font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font><font color='#979000'>10.0</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>3</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>3</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#009900'>// a simple function with a minimum at zero
</font>    <font color='#0000FF'><u>double</u></font> <b><a name='single_variable_function'></a>single_variable_function</b> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>double</u></font> x<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>total_count;
        <font color='#0000FF'>return</font> <font color='#979000'>3</font><font color='#5555FF'>*</font>x<font color='#5555FF'>*</font>x <font color='#5555FF'>+</font> <font color='#979000'>5</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font><font color='#009900'>// ----------------------------------------------------------------------------------------
</font><font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_apq'></a>test_apq</b> <font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>0</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> p
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>typedef</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>0</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> T;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>12</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> minf <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>10</font>;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>0</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>p.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>opt</font><font face='Lucida Console'>(</font>p.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>set_all_elements</font><font face='Lucida Console'>(</font>opt, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'><u>double</u></font> val <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>p.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>20</font><font face='Lucida Console'>)</font>
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>testing with apq and the start point: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>else</font>
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>testing with apq and a big vector with </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> p.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> components.</font>";

        <font color='#009900'>// don't use bfgs on really large vectors
</font>        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>p.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>20</font><font face='Lucida Console'>)</font>
        <b>{</b>
            total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            x <font color='#5555FF'>=</font> p;
            val <font color='#5555FF'>=</font> <font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>bfgs_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, 
                     <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                     <font color='#BB00BB'>wrap_function</font><font face='Lucida Console'>(</font>apq<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>wrap_function</font><font face='Lucida Console'>(</font>der_apq<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>)</font>, x, minf<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>apq</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() bgfs: got apq in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;

            total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            x <font color='#5555FF'>=</font> p;
            <font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>bfgs_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, 
                     <font color='#BB00BB'>gradient_norm_stop_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                     <font color='#BB00BB'>wrap_function</font><font face='Lucida Console'>(</font>apq<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>wrap_function</font><font face='Lucida Console'>(</font>der_apq<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>)</font>, x, minf<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() bgfs(gn): got apq in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;
        <b>}</b>


        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>p.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>100</font><font face='Lucida Console'>)</font>
        <b>{</b>
            total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            x <font color='#5555FF'>=</font> p;
            val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min_bobyqa</font><font face='Lucida Console'>(</font><font color='#BB00BB'>wrap_function</font><font face='Lucida Console'>(</font>apq<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>)</font>, x, <font color='#979000'>2</font><font color='#5555FF'>*</font>x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                            uniform_matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#5555FF'>-</font><font color='#979000'>1e100</font><font face='Lucida Console'>)</font>,
                            uniform_matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#979000'>1e100</font><font face='Lucida Console'>)</font>,
                            <font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>10</font>,
                            <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>,
                            <font color='#979000'>10000</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>apq</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min_bobyqa(): got apq in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;
        <b>}</b>

        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>lbfgs_search_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>10</font><font face='Lucida Console'>)</font>, 
                 <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                 <font color='#BB00BB'>wrap_function</font><font face='Lucida Console'>(</font>apq<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>wrap_function</font><font face='Lucida Console'>(</font>der_apq<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>)</font>, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>apq</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() lbgfs-10: got apq in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>lbfgs_search_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>, 
                 <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                 <font color='#BB00BB'>wrap_function</font><font face='Lucida Console'>(</font>apq<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>wrap_function</font><font face='Lucida Console'>(</font>der_apq<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>)</font>, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>apq</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() lbgfs-1: got apq in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>cg_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                 <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                 <font color='#BB00BB'>wrap_function</font><font face='Lucida Console'>(</font>apq<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>wrap_function</font><font face='Lucida Console'>(</font>der_apq<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>)</font>, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>apq</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() cg: got apq in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        <font color='#009900'>// don't do approximate derivative tests if the input point is really long
</font>        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>p.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>20</font><font face='Lucida Console'>)</font>
        <b>{</b>
            total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            x <font color='#5555FF'>=</font> p;
            val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>bfgs_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                     <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                     <font color='#BB00BB'>wrap_function</font><font face='Lucida Console'>(</font>apq<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font><font color='#BB00BB'>wrap_function</font><font face='Lucida Console'>(</font>apq<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, x, minf<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>apq</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() bfgs: got apq/noder in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


            total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            x <font color='#5555FF'>=</font> p;
            val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>cg_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                     <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                     <font color='#BB00BB'>wrap_function</font><font face='Lucida Console'>(</font>apq<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font><font color='#BB00BB'>wrap_function</font><font face='Lucida Console'>(</font>apq<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, x, minf<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>apq</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() cg: got apq/noder in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


            total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            x <font color='#5555FF'>=</font> p;
            val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min_using_approximate_derivatives</font><font face='Lucida Console'>(</font><font color='#BB00BB'>bfgs_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                                   <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>, 
                                                   <font color='#BB00BB'>wrap_function</font><font face='Lucida Console'>(</font>apq<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>)</font>, x, minf<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>apq</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() bfgs: got apq/noder2 in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


            total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            x <font color='#5555FF'>=</font> p;
            val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min_using_approximate_derivatives</font><font face='Lucida Console'>(</font><font color='#BB00BB'>lbfgs_search_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>10</font><font face='Lucida Console'>)</font>,
                                                   <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>, 
                                                   <font color='#BB00BB'>wrap_function</font><font face='Lucida Console'>(</font>apq<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>)</font>, x, minf<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() lbfgs-10: got apq/noder2 in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


            total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            x <font color='#5555FF'>=</font> p;
            val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min_using_approximate_derivatives</font><font face='Lucida Console'>(</font><font color='#BB00BB'>cg_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                                   <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                                                   <font color='#BB00BB'>wrap_function</font><font face='Lucida Console'>(</font>apq<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>)</font>, x, minf<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>apq</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() cg: got apq/noder2 in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;
        <b>}</b>
    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_powell'></a>test_powell</b> <font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>4</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> p
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>15</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> minf <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>1</font>;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>4</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> x, opt;
        <font color='#BB00BB'>opt</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#BB00BB'>opt</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#BB00BB'>opt</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#BB00BB'>opt</font><font face='Lucida Console'>(</font><font color='#979000'>3</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        <font color='#0000FF'><u>double</u></font> val <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>testing with powell and the start point: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font>;

        <font color='#009900'>/*
        total_count = 0;
        x = p;
        val=find_min(bfgs_search_strategy(),
                 objective_delta_stop_strategy(eps),
                 powell, derivative(powell,1e-8), x, minf);
        DLIB_TEST_MSG(dlib::equal(x,opt, 1e-2),opt-x);
        DLIB_TEST(approx_equal(val , powell(x)));
        dlog &lt;&lt; LINFO &lt;&lt; "find_min() bfgs: got powell/noder in " &lt;&lt; total_count;


        total_count = 0;
        x = p;
        val=find_min(cg_search_strategy(),
                 objective_delta_stop_strategy(eps),
                 powell, derivative(powell,1e-9), x, minf);
        DLIB_TEST_MSG(dlib::equal(x,opt, 1e-2),opt-x);
        DLIB_TEST(approx_equal(val , powell(x)));
        dlog &lt;&lt; LINFO &lt;&lt; "find_min() cg: got powell/noder in " &lt;&lt; total_count;
        */</font>

        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min_using_approximate_derivatives</font><font face='Lucida Console'>(</font><font color='#BB00BB'>bfgs_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                               <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                                               powell, x, minf, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>10</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>powell</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() bfgs: got powell/noder2 in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min_using_approximate_derivatives</font><font face='Lucida Console'>(</font><font color='#BB00BB'>lbfgs_search_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>,
                                               <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                                               powell, x, minf, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>10</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>powell</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() lbfgs-4: got powell/noder2 in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min_using_approximate_derivatives</font><font face='Lucida Console'>(</font><font color='#BB00BB'>lbfgs_search_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>,
                                               <font color='#BB00BB'>gradient_norm_stop_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                               powell, x, minf, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>10</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>powell</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() lbfgs-4(gn): got powell/noder2 in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min_using_approximate_derivatives</font><font face='Lucida Console'>(</font><font color='#BB00BB'>cg_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                               <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                                               powell, x, minf, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>10</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>powell</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() cg: got powell/noder2 in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min_bobyqa</font><font face='Lucida Console'>(</font>powell, x, <font color='#979000'>2</font><font color='#5555FF'>*</font>x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                        uniform_matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#5555FF'>-</font><font color='#979000'>1e100</font><font face='Lucida Console'>)</font>,
                        uniform_matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#979000'>1e100</font><font face='Lucida Console'>)</font>,
                        <font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>10</font>,
                        <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>8</font>,
                        <font color='#979000'>10000</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>3</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>powell</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min_bobyqa(): got powell in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;

    <b>}</b>



    <font color='#0000FF'><u>void</u></font> <b><a name='test_simple'></a>test_simple</b> <font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> p
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>12</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> minf <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>10000</font>;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> x, opt;
        <font color='#BB00BB'>opt</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#BB00BB'>opt</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#0000FF'><u>double</u></font> val <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>testing with simple and the start point: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font>;

        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>bfgs_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                 <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                 simple, der_simple, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>simple</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() bfgs: got simple in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>bfgs_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                 <font color='#BB00BB'>gradient_norm_stop_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                 simple, der_simple, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>simple</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() bfgs(gn): got simple in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>lbfgs_search_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>3</font><font face='Lucida Console'>)</font>,
                 <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                 simple, der_simple, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>simple</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() lbfgs-3: got simple in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>cg_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                 <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                 simple, der_simple, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>simple</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() cg: got simple in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;



        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>bfgs_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                 <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                 simple, <font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font>simple<font face='Lucida Console'>)</font>, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>simple</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() bfgs: got simple/noder in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>lbfgs_search_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>8</font><font face='Lucida Console'>)</font>,
                 <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                 simple, <font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font>simple<font face='Lucida Console'>)</font>, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>simple</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() lbfgs-8: got simple/noder in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>cg_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                 <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                 simple, <font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font>simple<font face='Lucida Console'>)</font>, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>simple</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() cg: got simple/noder in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;



        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min_using_approximate_derivatives</font><font face='Lucida Console'>(</font><font color='#BB00BB'>bfgs_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                               <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                                               simple, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>simple</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() bfgs: got simple/noder2 in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min_using_approximate_derivatives</font><font face='Lucida Console'>(</font><font color='#BB00BB'>lbfgs_search_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>,
                                               <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                                               simple, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>simple</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() lbfgs-6: got simple/noder2 in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min_using_approximate_derivatives</font><font face='Lucida Console'>(</font><font color='#BB00BB'>cg_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                               <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                                               simple, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>simple</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() cg: got simple/noder2 in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min_bobyqa</font><font face='Lucida Console'>(</font>simple, x, <font color='#979000'>2</font><font color='#5555FF'>*</font>x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                        uniform_matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#5555FF'>-</font><font color='#979000'>1e100</font><font face='Lucida Console'>)</font>,
                        uniform_matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#979000'>1e100</font><font face='Lucida Console'>)</font>,
                        <font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>10</font>,
                        <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>,
                        <font color='#979000'>10000</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>simple</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min_bobyqa(): got simple in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;

    <b>}</b>


    <font color='#0000FF'><u>void</u></font> <b><a name='test_rosen'></a>test_rosen</b> <font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> p
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>15</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> minf <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>10</font>;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> x, opt;
        <font color='#BB00BB'>opt</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        <font color='#BB00BB'>opt</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;

        <font color='#0000FF'><u>double</u></font> val <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>testing with rosen and the start point: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font>;

        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>bfgs_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                 <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                 rosen, der_rosen, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>7</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() bfgs: got rosen in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>bfgs_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                 <font color='#BB00BB'>gradient_norm_stop_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                 rosen, der_rosen, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>7</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() bfgs(gn): got rosen in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>lbfgs_search_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>20</font><font face='Lucida Console'>)</font>,
                 <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                 rosen, der_rosen, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>7</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() lbfgs-20: got rosen in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>cg_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                 <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                 rosen, der_rosen, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>7</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() cg: got rosen in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;



        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>bfgs_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                 <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                 rosen, <font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font>rosen,<font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() bfgs: got rosen/noder in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>lbfgs_search_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,
                 <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                 rosen, <font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font>rosen,<font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() lbfgs-5: got rosen/noder in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>cg_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                 <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                 rosen, <font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font>rosen,<font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() cg: got rosen/noder in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min_using_approximate_derivatives</font><font face='Lucida Console'>(</font><font color='#BB00BB'>cg_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                               <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>,
                                               rosen, x, minf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min() cg: got rosen/noder2 in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1000</font><font face='Lucida Console'>)</font>
        <b>{</b>
            total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            x <font color='#5555FF'>=</font> p;
            val<font color='#5555FF'>=</font><font color='#BB00BB'>find_min_bobyqa</font><font face='Lucida Console'>(</font>rosen, x, <font color='#979000'>2</font><font color='#5555FF'>*</font>x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                            uniform_matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#5555FF'>-</font><font color='#979000'>1e100</font><font face='Lucida Console'>)</font>,
                            uniform_matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#979000'>1e100</font><font face='Lucida Console'>)</font>,
                            <font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>10</font>,
                            <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>,
                            <font color='#979000'>10000</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min_bobyqa(): got rosen in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;
        <b>}</b>
    <b>}</b>


    <font color='#0000FF'><u>void</u></font> <b><a name='test_neg_rosen'></a>test_neg_rosen</b> <font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> p
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>15</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> maxf <font color='#5555FF'>=</font> <font color='#979000'>10</font>;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> x, opt;
        <font color='#BB00BB'>opt</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        <font color='#BB00BB'>opt</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;

        <font color='#0000FF'><u>double</u></font> val <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>testing with neg_rosen and the start point: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font>;

        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_max</font><font face='Lucida Console'>(</font>
            <font color='#BB00BB'>bfgs_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, 
            <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>, neg_rosen, der_neg_rosen, x, maxf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>7</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>neg_rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_max() bfgs: got neg_rosen in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;

        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_max</font><font face='Lucida Console'>(</font>
            <font color='#BB00BB'>lbfgs_search_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>, 
            <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>, neg_rosen, der_neg_rosen, x, maxf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>7</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>neg_rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_max() lbfgs-5: got neg_rosen in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;

        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_max</font><font face='Lucida Console'>(</font>
            <font color='#BB00BB'>lbfgs_search_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>, 
            <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>, neg_rosen, <font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font>neg_rosen<font face='Lucida Console'>)</font>, x, maxf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>7</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>neg_rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_max() lbfgs-5: got neg_rosen/noder in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_max_using_approximate_derivatives</font><font face='Lucida Console'>(</font>
            <font color='#BB00BB'>cg_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, 
            <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font>, neg_rosen, x, maxf<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>7</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>neg_rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_max() cg: got neg_rosen/noder2 in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        val<font color='#5555FF'>=</font><font color='#BB00BB'>find_max_bobyqa</font><font face='Lucida Console'>(</font>neg_rosen, x, <font color='#979000'>2</font><font color='#5555FF'>*</font>x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                        uniform_matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#5555FF'>-</font><font color='#979000'>1e100</font><font face='Lucida Console'>)</font>,
                        uniform_matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#979000'>1e100</font><font face='Lucida Console'>)</font>,
                        <font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>10</font>,
                        <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>,
                        <font color='#979000'>10000</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font>x,opt, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>,opt<font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>approx_equal</font><font face='Lucida Console'>(</font>val , <font color='#BB00BB'>neg_rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_max_bobyqa(): got neg_rosen in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_single_variable_function'></a>test_single_variable_function</b> <font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> p
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>7</font>;


        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>testing with single_variable_function and the start point: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> p;
        <font color='#0000FF'><u>double</u></font> out, x;

        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        out <font color='#5555FF'>=</font> <font color='#BB00BB'>find_min_single_variable</font><font face='Lucida Console'>(</font>single_variable_function, x, <font color='#5555FF'>-</font><font color='#979000'>1e100</font>, <font color='#979000'>1e100</font>, eps, <font color='#979000'>1000</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>out<font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, out<font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, x<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min_single_variable(): got single_variable_function in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        x <font color='#5555FF'>=</font> p;
        out <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#BB00BB'>find_max_single_variable</font><font face='Lucida Console'>(</font><font color='#BB00BB'>negate_function</font><font face='Lucida Console'>(</font>single_variable_function<font face='Lucida Console'>)</font>, x, <font color='#5555FF'>-</font><font color='#979000'>1e100</font>, <font color='#979000'>1e100</font>, eps, <font color='#979000'>1000</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>out<font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, out<font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, x<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_max_single_variable(): got single_variable_function in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>p <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
        <b>{</b>
            total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            x <font color='#5555FF'>=</font> p;
            out <font color='#5555FF'>=</font> <font color='#BB00BB'>find_min_single_variable</font><font face='Lucida Console'>(</font>single_variable_function, x, <font color='#5555FF'>-</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font>, <font color='#979000'>1e100</font>, eps, <font color='#979000'>1000</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>out<font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, out<font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, x<font face='Lucida Console'>)</font>;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min_single_variable(): got single_variable_function in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;


            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>p <font color='#5555FF'>&gt;</font> <font color='#979000'>3</font><font face='Lucida Console'>)</font>
            <b>{</b>
                total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                x <font color='#5555FF'>=</font> p;
                out <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#BB00BB'>find_max_single_variable</font><font face='Lucida Console'>(</font><font color='#BB00BB'>negate_function</font><font face='Lucida Console'>(</font>single_variable_function<font face='Lucida Console'>)</font>, x, <font color='#979000'>3</font>, <font color='#979000'>1e100</font>, eps, <font color='#979000'>1000</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>out <font color='#5555FF'>-</font> <font face='Lucida Console'>(</font><font color='#979000'>3</font><font color='#5555FF'>*</font><font color='#979000'>3</font><font color='#5555FF'>*</font><font color='#979000'>3</font><font color='#5555FF'>+</font><font color='#979000'>5</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, out<font color='#5555FF'>-</font><font face='Lucida Console'>(</font><font color='#979000'>3</font><font color='#5555FF'>*</font><font color='#979000'>3</font><font color='#5555FF'>*</font><font color='#979000'>3</font><font color='#5555FF'>+</font><font color='#979000'>5</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>x<font color='#5555FF'>-</font><font color='#979000'>3</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, x<font face='Lucida Console'>)</font>;
                dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_max_single_variable(): got single_variable_function in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>p <font color='#5555FF'>&lt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
        <b>{</b>
            total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            x <font color='#5555FF'>=</font> p;
            out <font color='#5555FF'>=</font> <font color='#BB00BB'>find_min_single_variable</font><font face='Lucida Console'>(</font>single_variable_function, x, <font color='#5555FF'>-</font><font color='#979000'>1e100</font>, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font>, eps, <font color='#979000'>1000</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>out<font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, out<font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, x<font face='Lucida Console'>)</font>;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min_single_variable(): got single_variable_function in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>p <font color='#5555FF'>&lt;</font> <font color='#5555FF'>-</font><font color='#979000'>3</font><font face='Lucida Console'>)</font>
            <b>{</b>
                total_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                x <font color='#5555FF'>=</font> p;
                out <font color='#5555FF'>=</font> <font color='#BB00BB'>find_min_single_variable</font><font face='Lucida Console'>(</font>single_variable_function, x, <font color='#5555FF'>-</font><font color='#979000'>1e100</font>, <font color='#5555FF'>-</font><font color='#979000'>3</font>, eps, <font color='#979000'>1000</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>out <font color='#5555FF'>-</font> <font face='Lucida Console'>(</font><font color='#979000'>3</font><font color='#5555FF'>*</font><font color='#979000'>3</font><font color='#5555FF'>*</font><font color='#979000'>3</font><font color='#5555FF'>+</font><font color='#979000'>5</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, out<font color='#5555FF'>-</font><font face='Lucida Console'>(</font><font color='#979000'>3</font><font color='#5555FF'>*</font><font color='#979000'>3</font><font color='#5555FF'>*</font><font color='#979000'>3</font><font color='#5555FF'>+</font><font color='#979000'>5</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>x<font color='#5555FF'>+</font><font color='#979000'>3</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, x<font face='Lucida Console'>)</font>;
                dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>find_min_single_variable(): got single_variable_function in </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> total_count;
            <b>}</b>
        <b>}</b>

    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='optimization_test'></a>optimization_test</b> <font face='Lucida Console'>(</font>
    <font face='Lucida Console'>)</font>
    <font color='#009900'>/*!
        ensures
            - runs tests on the optimization stuff compliance with the specs
    !*/</font>
    <b>{</b>        
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>0</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> p;

        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        p.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font face='Lucida Console'>)</font>;

        <font color='#009900'>// test with single_variable_function
</font>        <font color='#BB00BB'>test_single_variable_function</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>test_single_variable_function</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>test_single_variable_function</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font><font color='#979000'>10</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>test_single_variable_function</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font><font color='#979000'>100</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>test_single_variable_function</font><font face='Lucida Console'>(</font><font color='#979000'>900.53</font><font face='Lucida Console'>)</font>;

        <font color='#009900'>// test with the rosen function
</font>        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>9</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>4.9</font>;
        <font color='#BB00BB'>test_rosen</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>test_neg_rosen</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#BB00BB'>test_rosen</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>5323</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>98248</font>;
        <font color='#BB00BB'>test_rosen</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font>;

        <font color='#009900'>// test with the simple function
</font>        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        <font color='#BB00BB'>test_simple</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0.5</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>9</font>;
        <font color='#BB00BB'>test_simple</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>645</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>839485</font>;
        <font color='#BB00BB'>test_simple</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        <font color='#009900'>// test with the apq function
</font>        p.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>3</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>4</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        <font color='#BB00BB'>test_apq</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>3</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>4</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>5</font>;
        <font color='#BB00BB'>test_apq</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>3</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>3</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>4</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>5</font>;
        <font color='#BB00BB'>test_apq</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>2324</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>3</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>3</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>4</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>534534</font>;
        <font color='#BB00BB'>test_apq</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font>;

        p.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>10</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>3</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>3</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>4</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>5</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>5</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>6</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>7</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>3</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>8</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>9</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>5</font>;
        <font color='#BB00BB'>test_apq</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font>;

        <font color='#009900'>// test apq with a big vector
</font>        p.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>500</font><font face='Lucida Console'>)</font>;
        dlib::rand rnd;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> p.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#979000'>20</font> <font color='#5555FF'>-</font> <font color='#979000'>10</font>; 
        <b>}</b>
        <font color='#BB00BB'>test_apq</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        <font color='#009900'>// test with the powell function
</font>        p.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>1</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>3</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        <font color='#BB00BB'>test_powell</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font>;

        <b>{</b>
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> m;
            <font color='#BB00BB'>m</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>0.43</font>;
            <font color='#BB00BB'>m</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0.919</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>der_rosen</font><font face='Lucida Console'>(</font>m<font face='Lucida Console'>)</font> , <font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font>rosen<font face='Lucida Console'>)</font><font face='Lucida Console'>(</font>m<font face='Lucida Console'>)</font>,<font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font><font color='#BB00BB'>make_line_search_function</font><font face='Lucida Console'>(</font>rosen,m,m<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> 
                                  <font color='#BB00BB'>make_line_search_function</font><font face='Lucida Console'>(</font><font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font>rosen<font face='Lucida Console'>)</font>,m,m<font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>,"<font color='#CC0000'></font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font><font color='#BB00BB'>make_line_search_function</font><font face='Lucida Console'>(</font>rosen,m,m<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> 
                                  <font color='#BB00BB'>make_line_search_function</font><font face='Lucida Console'>(</font><font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font>rosen<font face='Lucida Console'>)</font>,m,m<font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>,"<font color='#CC0000'></font>"<font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font><font color='#BB00BB'>make_line_search_function</font><font face='Lucida Console'>(</font>rosen,m,m<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> 
                                  <font color='#BB00BB'>make_line_search_function</font><font face='Lucida Console'>(</font>der_rosen,m,m<font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>,"<font color='#CC0000'></font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font><font color='#BB00BB'>make_line_search_function</font><font face='Lucida Console'>(</font>rosen,m,m<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> 
                                  <font color='#BB00BB'>make_line_search_function</font><font face='Lucida Console'>(</font>der_rosen,m,m<font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>,"<font color='#CC0000'></font>"<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> m;
            <font color='#BB00BB'>m</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            <font color='#BB00BB'>m</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>dlib::<font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>der_rosen</font><font face='Lucida Console'>(</font>m<font face='Lucida Console'>)</font> , <font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font>rosen<font face='Lucida Console'>)</font><font face='Lucida Console'>(</font>m<font face='Lucida Console'>)</font>,<font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font><font color='#BB00BB'>make_line_search_function</font><font face='Lucida Console'>(</font>rosen,m,m<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> 
                                  <font color='#BB00BB'>make_line_search_function</font><font face='Lucida Console'>(</font><font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font>rosen<font face='Lucida Console'>)</font>,m,m<font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>,"<font color='#CC0000'></font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font><font color='#BB00BB'>make_line_search_function</font><font face='Lucida Console'>(</font>rosen,m,m<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> 
                                  <font color='#BB00BB'>make_line_search_function</font><font face='Lucida Console'>(</font><font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font>rosen<font face='Lucida Console'>)</font>,m,m<font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>,"<font color='#CC0000'></font>"<font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font><font color='#BB00BB'>make_line_search_function</font><font face='Lucida Console'>(</font>rosen,m,m<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> 
                                  <font color='#BB00BB'>make_line_search_function</font><font face='Lucida Console'>(</font>der_rosen,m,m<font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>,"<font color='#CC0000'></font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font><font color='#BB00BB'>make_line_search_function</font><font face='Lucida Console'>(</font>rosen,m,m<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> 
                                  <font color='#BB00BB'>make_line_search_function</font><font face='Lucida Console'>(</font>der_rosen,m,m<font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>,"<font color='#CC0000'></font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <b>{</b>
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> m;
            m <font color='#5555FF'>=</font> <font color='#979000'>1</font>,<font color='#979000'>2</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>neg_rosen</font><font face='Lucida Console'>(</font>m<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>negate_function</font><font face='Lucida Console'>(</font>rosen<font face='Lucida Console'>)</font><font face='Lucida Console'>(</font>m<font face='Lucida Console'>)</font> <font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>16</font><font face='Lucida Console'>)</font>;
        <b>}</b>

    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> der_funct, <font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'><u>double</u></font> <b><a name='unconstrained_gradient_magnitude'></a>unconstrained_gradient_magnitude</b> <font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> der_funct<font color='#5555FF'>&amp;</font> grad,
        <font color='#0000FF'>const</font> T<font color='#5555FF'>&amp;</font> x,
        <font color='#0000FF'>const</font> T<font color='#5555FF'>&amp;</font> lower,
        <font color='#0000FF'>const</font> T<font color='#5555FF'>&amp;</font> upper
    <font face='Lucida Console'>)</font>
    <b>{</b>
        T g <font color='#5555FF'>=</font> <font color='#BB00BB'>grad</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;

        <font color='#0000FF'><u>double</u></font> unorm <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> g.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                unorm <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#BB00BB'>g</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>g</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> <font color='#BB00BB'>g</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                unorm <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#BB00BB'>g</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>g</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> <font color='#BB00BB'>g</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                unorm <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#BB00BB'>g</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>g</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>return</font> unorm;
    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> der_funct, <font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'><u>double</u></font> <b><a name='unconstrained_gradient_magnitude_neg_funct'></a>unconstrained_gradient_magnitude_neg_funct</b> <font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> der_funct<font color='#5555FF'>&amp;</font> grad,
        <font color='#0000FF'>const</font> T<font color='#5555FF'>&amp;</font> x,
        <font color='#0000FF'>const</font> T<font color='#5555FF'>&amp;</font> lower,
        <font color='#0000FF'>const</font> T<font color='#5555FF'>&amp;</font> upper
    <font face='Lucida Console'>)</font>
    <b>{</b>
        T g <font color='#5555FF'>=</font> <font color='#BB00BB'>grad</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;

        <font color='#0000FF'><u>double</u></font> unorm <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> g.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                unorm <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#BB00BB'>g</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>g</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> <font color='#BB00BB'>g</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                unorm <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#BB00BB'>g</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>g</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> <font color='#BB00BB'>g</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                unorm <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#BB00BB'>g</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>g</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>return</font> unorm;
    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> search_strategy_type<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'><u>double</u></font> <b><a name='test_bound_solver_neg_rosen'></a>test_bound_solver_neg_rosen</b> <font face='Lucida Console'>(</font>dlib::rand<font color='#5555FF'>&amp;</font> rnd, search_strategy_type search_strategy<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::test_functions;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> starting_point, lower, upper, x;


        <font color='#009900'>// pick random bounds
</font>        lower <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font>, rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font>;
        upper <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font>, rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font>;
        <font color='#0000FF'>while</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font>;
        <font color='#0000FF'>while</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font>;

        starting_point <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>, 
                       rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>lower: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>lower<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>upper: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>upper<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>starting: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>starting_point<font face='Lucida Console'>)</font>;

        x <font color='#5555FF'>=</font> starting_point;
        <font color='#0000FF'><u>double</u></font> val <font color='#5555FF'>=</font> <font color='#BB00BB'>find_max_box_constrained</font><font face='Lucida Console'>(</font> 
            search_strategy,
            <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>16</font>, <font color='#979000'>500</font><font face='Lucida Console'>)</font>, 
            neg_rosen, der_neg_rosen, x,
            lower,  
            upper   
        <font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>val <font color='#5555FF'>-</font> <font color='#BB00BB'>neg_rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>11</font>, std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>val <font color='#5555FF'>-</font> <font color='#BB00BB'>neg_rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>neg_rosen solution:\n</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> x;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>neg_rosen gradient: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font><font color='#BB00BB'>der_neg_rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> gradient_residual <font color='#5555FF'>=</font> <font color='#BB00BB'>unconstrained_gradient_magnitude_neg_funct</font><font face='Lucida Console'>(</font>der_neg_rosen, x, lower, upper<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>gradient_residual: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> gradient_residual;

        <font color='#0000FF'>return</font> gradient_residual;
    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> search_strategy_type<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'><u>double</u></font> <b><a name='test_bound_solver_rosen'></a>test_bound_solver_rosen</b> <font face='Lucida Console'>(</font>dlib::rand<font color='#5555FF'>&amp;</font> rnd, search_strategy_type search_strategy<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::test_functions;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> starting_point, lower, upper, x;


        <font color='#009900'>// pick random bounds and sometimes put the upper bound at zero so we can have
</font>        <font color='#009900'>// a test where the optimal value has a bound active at 0 so make sure this case
</font>        <font color='#009900'>// works properly.
</font>        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0.2</font><font face='Lucida Console'>)</font>
        <b>{</b>
            lower <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font>, rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font>;
            upper <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font>, rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font>;
            <font color='#0000FF'>while</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font>;
            <font color='#0000FF'>while</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font>;
        <b>}</b>
        <font color='#0000FF'>else</font>
        <b>{</b>
            upper <font color='#5555FF'>=</font> <font color='#979000'>0</font>,<font color='#979000'>0</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0.5</font><font face='Lucida Console'>)</font>
                <font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0.5</font><font face='Lucida Console'>)</font>
                <font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            lower <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font>, rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font>;
            lower <font color='#5555FF'>=</font> upper <font color='#5555FF'>-</font> lower;
        <b>}</b>
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>bool</u></font> pick_uniform_bounds <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0.9</font>;
        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>pick_uniform_bounds<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>double</u></font> x <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#979000'>2</font>;
            <font color='#0000FF'><u>double</u></font> y <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#979000'>2</font>;
            lower <font color='#5555FF'>=</font> <font color='#BB00BB'>min</font><font face='Lucida Console'>(</font>x,y<font face='Lucida Console'>)</font>;
            upper <font color='#5555FF'>=</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>x,y<font face='Lucida Console'>)</font>;
        <b>}</b>

        starting_point <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>, 
                       rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>lower: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>lower<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>upper: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>upper<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>starting: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>starting_point<font face='Lucida Console'>)</font>;

        x <font color='#5555FF'>=</font> starting_point;
        <font color='#0000FF'><u>double</u></font> val;
        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>pick_uniform_bounds<font face='Lucida Console'>)</font>
        <b>{</b>
            val <font color='#5555FF'>=</font> <font color='#BB00BB'>find_min_box_constrained</font><font face='Lucida Console'>(</font> 
                search_strategy,
                <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>16</font>, <font color='#979000'>500</font><font face='Lucida Console'>)</font>, 
                rosen, der_rosen, x,
                lower,  
                upper   
            <font face='Lucida Console'>)</font>;
        <b>}</b>
        <font color='#0000FF'>else</font>
        <b>{</b>
            val <font color='#5555FF'>=</font> <font color='#BB00BB'>find_min_box_constrained</font><font face='Lucida Console'>(</font> 
                search_strategy,
                <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>16</font>, <font color='#979000'>500</font><font face='Lucida Console'>)</font>, 
                rosen, der_rosen, x,
                <font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,  
                <font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>   
            <font face='Lucida Console'>)</font>;
        <b>}</b>


        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>val <font color='#5555FF'>-</font> <font color='#BB00BB'>rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>11</font>, std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>val <font color='#5555FF'>-</font> <font color='#BB00BB'>rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>rosen solution:\n</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> x;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>rosen gradient: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font><font color='#BB00BB'>der_rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> gradient_residual <font color='#5555FF'>=</font> <font color='#BB00BB'>unconstrained_gradient_magnitude</font><font face='Lucida Console'>(</font>der_rosen, x, lower, upper<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>gradient_residual: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> gradient_residual;

        <font color='#0000FF'>return</font> gradient_residual;
    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> search_strategy_type<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'><u>double</u></font> <b><a name='test_bound_solver_brown'></a>test_bound_solver_brown</b> <font face='Lucida Console'>(</font>dlib::rand<font color='#5555FF'>&amp;</font> rnd, search_strategy_type search_strategy<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::test_functions;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>4</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>starting_point</font><font face='Lucida Console'>(</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>, x;

        <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>0</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> solution <font color='#5555FF'>=</font> <font color='#BB00BB'>brown_solution</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        <font color='#009900'>// pick random bounds
</font>        lower <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        lower <font color='#5555FF'>=</font> lower<font color='#5555FF'>*</font><font color='#979000'>10</font> <font color='#5555FF'>+</font> solution;
        upper <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        upper <font color='#5555FF'>=</font> upper<font color='#5555FF'>*</font><font color='#979000'>10</font> <font color='#5555FF'>+</font> solution;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> lower.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> 
                <font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font><font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font>,<font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        starting_point <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>, 
                       rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
                       rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font face='Lucida Console'>)</font>,
                       rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#BB00BB'>upper</font><font face='Lucida Console'>(</font><font color='#979000'>3</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>3</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>lower</font><font face='Lucida Console'>(</font><font color='#979000'>3</font><font face='Lucida Console'>)</font>;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>lower: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>lower<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>upper: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>upper<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>starting: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>starting_point<font face='Lucida Console'>)</font>;

        x <font color='#5555FF'>=</font> starting_point;
        <font color='#0000FF'><u>double</u></font> val <font color='#5555FF'>=</font> <font color='#BB00BB'>find_min_box_constrained</font><font face='Lucida Console'>(</font> 
            search_strategy,
            <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>16</font>, <font color='#979000'>500</font><font face='Lucida Console'>)</font>, 
            brown, brown_derivative, x,
            lower,  
            upper   
        <font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>val <font color='#5555FF'>-</font> <font color='#BB00BB'>brown</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>14</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>brown solution:\n</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> x;
        <font color='#0000FF'>return</font> <font color='#BB00BB'>unconstrained_gradient_magnitude</font><font face='Lucida Console'>(</font>brown_derivative, x, lower, upper<font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> search_strategy_type<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_box_constrained_optimizers'></a>test_box_constrained_optimizers</b><font face='Lucida Console'>(</font>search_strategy_type search_strategy<font face='Lucida Console'>)</font>
    <b>{</b>
        dlib::rand rnd;
        running_stats<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> rs;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>test find_min_box_constrained() on rosen</font>";
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> <font color='#979000'>10000</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            rs.<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#BB00BB'>test_bound_solver_rosen</font><font face='Lucida Console'>(</font>rnd, search_strategy<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>mean rosen gradient: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> rs.<font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>max rosen gradient:  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> rs.<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>rs.<font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>12</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>rs.<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>9</font><font face='Lucida Console'>)</font>;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>test find_min_box_constrained() on brown</font>";
        rs.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> <font color='#979000'>1000</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            rs.<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#BB00BB'>test_bound_solver_brown</font><font face='Lucida Console'>(</font>rnd, search_strategy<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>mean brown gradient: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> rs.<font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>max brown gradient:  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> rs.<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>min brown gradient:  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> rs.<font color='#BB00BB'>min</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>rs.<font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>4e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>rs.<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>3e</font><font color='#5555FF'>-</font><font color='#979000'>2</font>, rs.<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>rs.<font color='#BB00BB'>min</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>10</font><font face='Lucida Console'>)</font>;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>test find_max_box_constrained() on neg_rosen</font>";
        rs.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> <font color='#979000'>1000</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            rs.<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#BB00BB'>test_bound_solver_neg_rosen</font><font face='Lucida Console'>(</font>rnd, search_strategy<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>mean neg_rosen gradient: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> rs.<font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>max neg_rosen gradient:  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> rs.<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>rs.<font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>12</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>rs.<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>9</font><font face='Lucida Console'>)</font>;

    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_poly_min_extract_2nd'></a>test_poly_min_extract_2nd</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'><u>double</u></font> off;

        off <font color='#5555FF'>=</font> <font color='#979000'>0.0</font>; <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font> <font color='#BB00BB'>poly_min_extrap</font><font face='Lucida Console'>(</font>off<font color='#5555FF'>*</font>off, <font color='#5555FF'>-</font><font color='#979000'>2</font><font color='#5555FF'>*</font>off, <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>off<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>off<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> off<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>13</font><font face='Lucida Console'>)</font>; 
        off <font color='#5555FF'>=</font> <font color='#979000'>0.1</font>; <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font> <font color='#BB00BB'>poly_min_extrap</font><font face='Lucida Console'>(</font>off<font color='#5555FF'>*</font>off, <font color='#5555FF'>-</font><font color='#979000'>2</font><font color='#5555FF'>*</font>off, <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>off<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>off<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> off<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>13</font><font face='Lucida Console'>)</font>; 
        off <font color='#5555FF'>=</font> <font color='#979000'>0.2</font>; <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font> <font color='#BB00BB'>poly_min_extrap</font><font face='Lucida Console'>(</font>off<font color='#5555FF'>*</font>off, <font color='#5555FF'>-</font><font color='#979000'>2</font><font color='#5555FF'>*</font>off, <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>off<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>off<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> off<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>13</font><font face='Lucida Console'>)</font>; 
        off <font color='#5555FF'>=</font> <font color='#979000'>0.3</font>; <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font> <font color='#BB00BB'>poly_min_extrap</font><font face='Lucida Console'>(</font>off<font color='#5555FF'>*</font>off, <font color='#5555FF'>-</font><font color='#979000'>2</font><font color='#5555FF'>*</font>off, <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>off<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>off<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> off<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>13</font><font face='Lucida Console'>)</font>; 
        off <font color='#5555FF'>=</font> <font color='#979000'>0.4</font>; <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font> <font color='#BB00BB'>poly_min_extrap</font><font face='Lucida Console'>(</font>off<font color='#5555FF'>*</font>off, <font color='#5555FF'>-</font><font color='#979000'>2</font><font color='#5555FF'>*</font>off, <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>off<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>off<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> off<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>13</font><font face='Lucida Console'>)</font>; 
        off <font color='#5555FF'>=</font> <font color='#979000'>0.5</font>; <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font> <font color='#BB00BB'>poly_min_extrap</font><font face='Lucida Console'>(</font>off<font color='#5555FF'>*</font>off, <font color='#5555FF'>-</font><font color='#979000'>2</font><font color='#5555FF'>*</font>off, <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>off<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>off<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> off<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>13</font><font face='Lucida Console'>)</font>; 
        off <font color='#5555FF'>=</font> <font color='#979000'>0.6</font>; <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font> <font color='#BB00BB'>poly_min_extrap</font><font face='Lucida Console'>(</font>off<font color='#5555FF'>*</font>off, <font color='#5555FF'>-</font><font color='#979000'>2</font><font color='#5555FF'>*</font>off, <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>off<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>off<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> off<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>13</font><font face='Lucida Console'>)</font>; 
        off <font color='#5555FF'>=</font> <font color='#979000'>0.8</font>; <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font> <font color='#BB00BB'>poly_min_extrap</font><font face='Lucida Console'>(</font>off<font color='#5555FF'>*</font>off, <font color='#5555FF'>-</font><font color='#979000'>2</font><font color='#5555FF'>*</font>off, <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>off<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>off<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> off<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>13</font><font face='Lucida Console'>)</font>; 
        off <font color='#5555FF'>=</font> <font color='#979000'>0.9</font>; <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font> <font color='#BB00BB'>poly_min_extrap</font><font face='Lucida Console'>(</font>off<font color='#5555FF'>*</font>off, <font color='#5555FF'>-</font><font color='#979000'>2</font><font color='#5555FF'>*</font>off, <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>off<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>off<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> off<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>13</font><font face='Lucida Console'>)</font>; 
        off <font color='#5555FF'>=</font> <font color='#979000'>1.0</font>; <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font> <font color='#BB00BB'>poly_min_extrap</font><font face='Lucida Console'>(</font>off<font color='#5555FF'>*</font>off, <font color='#5555FF'>-</font><font color='#979000'>2</font><font color='#5555FF'>*</font>off, <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>off<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>off<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> off<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>13</font><font face='Lucida Console'>)</font>; 
    <b>}</b>

    <font color='#0000FF'>class</font> <b><a name='optimization_tester'></a>optimization_tester</b> : <font color='#0000FF'>public</font> tester
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='optimization_tester'></a>optimization_tester</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> :
            tester <font face='Lucida Console'>(</font>"<font color='#CC0000'>test_optimization</font>",
                    "<font color='#CC0000'>Runs tests on the optimization component.</font>"<font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='perform_test'></a>perform_test</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>test_box_constrained_optimizers(bfgs_search_strategy())</font>";
            <font color='#BB00BB'>test_box_constrained_optimizers</font><font face='Lucida Console'>(</font><font color='#BB00BB'>bfgs_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>test_box_constrained_optimizers(lbfgs_search_strategy(5))</font>";
            <font color='#BB00BB'>test_box_constrained_optimizers</font><font face='Lucida Console'>(</font><font color='#BB00BB'>lbfgs_search_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>5</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_poly_min_extract_2nd</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>optimization_test</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>
    <b>}</b> a;

<b>}</b>



</pre></body></html>